package com.raisecom.crawler;

import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;

import com.raisecom.util.ChartSet;
import com.raisecom.util.HtmlParseUtil;

/**
 * 根据APP用户读书进行分类
 * 
 * @author sks
 * 
 */
public class JsoupCrawler {

	public static void main(String[] args) throws IOException, InterruptedException {
		JsoupCrawler acs = new JsoupCrawler();

		// 爬取“百度阅读”书籍分类信息
//		Map<String, String> allBookCateGory = acs.getAllBookCateGory("http://yuedu.baidu.com/book/list/0?show=0");
//		acs.getAllBookIdsBaseOnCateGory(allBookCateGory);

		// 爬取“起点读书”书籍分类信息  
		// http://free.qidian.com/   免费书籍
//		for (int i = 1; i <= 275; i++) {
//			Thread.sleep(20000);
//			acs.getAllBookInfoForFreeQiDian("http://free.qidian.com/Free/BookStore.aspx?total=275&pageIndex=" + i
//					+ "&dataCount=8231&orderType=9&categoryId=-1&wordsCount=-1&actionStatus=-1&firstLetter=all");
//		}
		
		// 爬取“起点读书”书籍分类信息  
		// http://www.qidian.com/Default.aspx 全部书籍
		for (int i = 0; i <= 1200; i++) {
			if (i % 20 == 0) {
				System.out.println("===================" + i + "=========================");
			}
			acs.getAllBookInfoForAllQiDian("http://all.qidian.com/Book/BookStore.aspx?ChannelId=-1&SubCategoryId=-1&Tag=all&Size=-1&Action=-1&OrderId=6&P=all&PageIndex=" + i
					+ "&update=-1&Vip=-1&Boutique=-1&SignStatus=-1");
//			Thread.sleep(1000);
		}

		// 爬取“塔读文学”书籍分类信息
//		for (int i = 1; i <= 398; i++) {
//			acs.getAllBookInfoForTaDu("http://www.tadu.com/book/list/?categoryId=&charNum=&countType=&timeType=&isSerial=&channelId=&page=" + i);
//			Thread.sleep(20000);
//		}

		// 爬取“QQ阅读”书籍分类信息
		// "QQ阅读"包括“创世中文网”，"云起书院"，“畅销图书”,需爬取三个
		// “创世中文网” part-1
//		for (int i = 1; i <= 1282; i++) {
//			acs.getAllBookInfoForQQYueduBaseOnCSZW("http://chuangshi.qq.com/bk/p/" + i + ".html");
//			Thread.sleep(30000);
//		}
		// "云起书院" part-2
//		for (int i = 1; i <= 2030; i++) {
//			acs.getAllBookInfoForQQYueduBaseOnYQSY("http://yunqi.qq.com/bk/so2/n30p" + i);
//			Thread.sleep(30000);
//		}
		// "畅销图书" part-3
//		for (int i = 1; i <= 470; i++) {
//			acs.getAllBookInfoForQQYueduBaseOnCXTS("http://dushu.qq.com/store/index/words/-1/finished/-1/feetype/-1/sortkey/2/ps/50/p/" + i);
//			Thread.sleep(30000);
//		}
	}

	/**
	 * 爬取“QQ阅读”书籍分类信息 基于“畅销图书”
	 * 
	 * @param url
	 * @throws IOException
	 */
	private void getAllBookInfoForQQYueduBaseOnCXTS(String url) throws IOException {
		String bookIdsDoc = HtmlParseUtil.getHtmlDoc(url, ChartSet.UTF_8);
		Document doc = Jsoup.parse(bookIdsDoc);

		Elements seletEles = doc.select("div.book");
		Iterator<Element> iterSeletEles = seletEles.iterator();
		while (iterSeletEles.hasNext()) {
			try {
				Element nextEle = iterSeletEles.next();
				String bookName = nextEle.child(1).child(0).child(1).childNode(0).toString();
				String cateGroy = nextEle.child(1).child(1).child(3).child(0).childNode(0).toString();
				String href = nextEle.child(1).child(0).child(0).child(0).attr("href");
				String bookCode = href.split("=")[1];
				System.out.println(cateGroy + "-" + bookName + "-" + bookCode);
			} catch (Exception e) {
				break;
			}
		}
	}

	/**
	 * 爬取"QQ阅读"书籍分类信息 基于“云起书院”
	 * 
	 * @param url
	 * @throws IOException
	 */
	private void getAllBookInfoForQQYueduBaseOnYQSY(String url) throws IOException {
		String bookIdsDoc = HtmlParseUtil.getHtmlDoc(url, ChartSet.UTF_8);
		Document doc = Jsoup.parse(bookIdsDoc);
		Elements bookInfoEles = doc.select("div.book");

		Iterator<Element> iterBookInfoEles = bookInfoEles.iterator();
		while (iterBookInfoEles.hasNext()) {
			try {
				Element nextBookInfoEle = iterBookInfoEles.next();
				String href = nextBookInfoEle.child(0).attr("href");
				String bookCode = href.substring(href.lastIndexOf("/") + 1, href.lastIndexOf("."));
				String bookName = nextBookInfoEle.child(1).child(0).child(1).childNode(0).toString();
				String fatCateGory = nextBookInfoEle.child(1).child(1).child(3).child(0).childNode(0).toString();
				String thinCateGroy = fatCateGory.substring(1, fatCateGory.length() - 1);
				System.out.println(thinCateGroy + "-" + bookName + "-" + bookCode);
			} catch (Exception e) {
				break;
			}
		}
	}

	/**
	 * 爬取"QQ阅读"书籍分类信息 基于“创世中文网”
	 * 
	 * @param url
	 * @throws IOException
	 */
	private void getAllBookInfoForQQYueduBaseOnCSZW(String url) throws IOException {
		String bookIdsDoc = HtmlParseUtil.getHtmlDoc(url, ChartSet.UTF_8);
		Document doc = Jsoup.parse(bookIdsDoc);
		Elements leftlistEles = doc.select("div.leftlist");
		
		Element bookRootEle = leftlistEles.get(0).child(1);
		Elements bookInfoEles = bookRootEle.child(0).children();

		for (int i = 1; i < bookInfoEles.size(); i++) {
			try {
				Element currEle = bookInfoEles.get(i);
				String bigCateGory = currEle.child(1).child(0).childNode(0).toString();
				String thinCateGory = bigCateGory.substring(1, bigCateGory.length() - 1);

				Element bookEle = currEle.child(2).child(0);
				String bookName = bookEle.childNode(0).toString();
				String herf = bookEle.attr("href");
				String bookCode = herf.substring(herf.lastIndexOf("/") + 1, herf.lastIndexOf("."));
				System.out.println(thinCateGory + "-" + bookName + "-" + bookCode);
			} catch (Exception e) {
				break;
			}
		}
	}

	/**
	 * 爬取“塔读文学”书籍分类信息
	 * 
	 * @param string
	 * @throws IOException
	 */
	private void getAllBookInfoForTaDu(String url) throws IOException {
		String bookIdsDoc = HtmlParseUtil.getHtmlDoc(url, ChartSet.UTF_8);
		Document doc = Jsoup.parse(bookIdsDoc);
		Elements eles = doc.select("#book_list");
		List<Node> childNodes = eles.get(0).child(1).childNodes();

		for (int i = 0; i < childNodes.size(); i++) {
			Node node = childNodes.get(i);
			if (!"".equals(node.toString().trim())) {
				try {
					String cateGoryName = node.childNode(3).childNode(1).attr("title");
					String bookName = node.childNode(5).childNode(1).attr("title");
					String href = node.childNode(5).childNode(1).attr("href");
					String bookCode = href.substring(6, href.length() - 1);
					System.out.println(cateGoryName + "-" + bookName + "-" + bookCode);
				} catch (Exception e) {
					break;
				}
			}
		}
	}

	/**
	 * 爬取"起点读书"分类信息
	 * 全部分书籍
	 * @param url
	 * @throws IOException
	 */
	private void getAllBookInfoForAllQiDian(String url) throws IOException {
		String bookIdsDoc = HtmlParseUtil.getHtmlDoc(url, ChartSet.UTF_8);
		Document doc = Jsoup.parse(bookIdsDoc);
		
		printAllBookInfoForAllQiDian(doc, "div.sw2");
		printAllBookInfoForAllQiDian(doc, "div.sw1");
	}

	/**
	 * 打印
	 * @param doc
	 * @param divType
	 */
	private void printAllBookInfoForAllQiDian(Document doc, String divType) {
		Iterator<Element> iterEles = doc.select(divType).iterator();
		
		while (iterEles.hasNext()) {
			try {
				Element nextEle = iterEles.next();
				Element cateGoryEle = nextEle.child(1);
				
				String parentCateGroy = cateGoryEle.child(0).childNode(0).toString();
				String sonCateGroy = cateGoryEle.child(1).childNode(0).toString();
				String cateGory = parentCateGroy + "/" + sonCateGroy;
				
				Element bookNameEle = nextEle.child(2);
				String bookName = bookNameEle.child(0).child(0).childNode(0).toString();
				
				Element bookCodeEle = nextEle.child(2);
				String href = bookCodeEle.child(0).child(0).attr("href");
				String bookCode = href.substring(href.lastIndexOf("/") + 1, href.lastIndexOf("."));
				String author = nextEle.child(4).child(0).childNode(0).toString();
				String updateTime = nextEle.child(5).childNode(0).toString();
				
//				System.out.println(updateTime);
				System.out.println(cateGory + "|" + bookName + "|" + bookCode + "|" + author + "|" + updateTime.substring(1));
			} catch (Exception e) {
				continue;
			}
		}
	}
	
	/**
	 * 爬取"起点读书"分类信息
	 * free部分书籍
	 * @param string
	 * @throws IOException
	 */
	private void getAllBookInfoForFreeQiDian(String url) throws IOException {
		String bookIdsDoc = HtmlParseUtil.getHtmlDoc(url, ChartSet.UTF_8);
		Document doc = Jsoup.parse(bookIdsDoc);
		Elements els = doc.select("div.book_list");
		Elements childrenEles = els.get(0).children();
		Elements eleByTags = childrenEles.get(0).getElementsByTag("tbody").get(0).getElementsByTag("tr");
		Iterator<Element> iterEles = eleByTags.iterator();

		while (iterEles.hasNext()) {
			try {
				Element nextEle = iterEles.next();

				String cateGoryName = nextEle.child(1).child(0).attr("title");
				Node bookName = nextEle.child(2).childNode(0).childNode(0);
				String herfBookId = nextEle.child(2).childNode(0).attr("href");
				String[] herfBookIds = herfBookId.split("=");
				System.out.println(cateGoryName + "-" + bookName + "-" + herfBookIds[1]);
			} catch (Exception e) {
				break;
			}
		}
	}

	/**
	 * 爬取所有百度阅读中的书籍名称及BookID
	 * 
	 * @param allBookCateGory
	 * @throws IOException
	 */
	private void getAllBookIdsBaseOnCateGory(Map<String, String> allBookCateGory) throws IOException {
		Iterator<Entry<String, String>> iterBookCateGorys = allBookCateGory.entrySet().iterator();

		while (iterBookCateGorys.hasNext()) {
			Entry<String, String> nextBookCateGory = iterBookCateGorys.next();

			String cateGoryName = nextBookCateGory.getKey();
			String cateGoryCode = nextBookCateGory.getValue();

			// System.out.println(cateGoryCode);
			getAllBookInfoBasedOnCode(cateGoryName, cateGoryCode);
		}
	}

	/**
	 * 基于书籍分类代号爬取BookIDs
	 * 
	 * @param cateGoryName
	 * @param cateGoryCode
	 * @throws IOException
	 */
	private void getAllBookInfoBasedOnCode(String cateGoryName, String cateGoryCode) throws IOException {
		try {
			String cateGroyUrl = "http://yuedu.baidu.com/book/list/" + cateGoryCode + "?od=0&show=0&pn=";

			for (int i = 0;; i += 20) {
				Thread.sleep(30000);

				String bookIdsDoc = HtmlParseUtil.getHtmlDoc(cateGroyUrl + i, ChartSet.GB2312);
				Document doc = Jsoup.parse(bookIdsDoc);
				Elements els = doc.select("div.book a[href^=/ebook/]");

				int size = els.size();

				if (size <= 0) {
					break;
				} else {
					for (int j = 0; j < size; j++) {
						try {
							Element ebookEle = els.get(j);
							String herf = ebookEle.attr("href");
							String bookName = ebookEle.child(0).attr("alt");

							String bookId = herf.substring(7, herf.indexOf("?"));

							if (bookName != null && !"".equals(bookName.trim())) {
								System.out.println(cateGoryName + "-" + cateGoryCode + "-" + bookId + "-" + bookName);
							}
						} catch (Exception e) {
							break;
						}
					}
				}
			}
		} catch (InterruptedException e) {
			System.out.println(e);
		}
	}

	/**
	 * 获取百度阅读书籍分类Code
	 * 
	 * @param url
	 * @throws IOException
	 */
	private Map<String, String> getAllBookCateGory(String url) throws IOException {
		Map<String, String> cateGoryMap = new HashMap<String, String>();

		String appCategroyHtml = HtmlParseUtil.getHtmlDoc(url, ChartSet.GB2312);
		Document doc = Jsoup.parse(appCategroyHtml);
		Elements els = doc.select("div.inner");

		Element firstEle = els.get(0);
		Iterator<Node> iterNodes = firstEle.childNodes().iterator();
		while (iterNodes.hasNext()) {
			Node nextNode = iterNodes.next();
			Node parentCateGoryNode = nextNode.childNode(0).childNode(0).childNode(0).childNode(0);
			// 父类分类名称
			// System.out.println(parentCateGoryNode);

			Node childNode = nextNode.childNode(1).childNode(0);
			Iterator<Node> iterChildNodes = childNode.childNodes().iterator();

			while (iterChildNodes.hasNext()) {
				Node nextChildNode = iterChildNodes.next();
				// System.out.println(nextChildNode.childNode(0));
				// System.out.println(parentCateGoryNode + "/" +
				// nextChildNode.childNode(0).attr("title"));
				// System.out.println(nextChildNode.childNode(0).attr("href").substring(11));

				String cateGoryName = parentCateGoryNode + "/" + nextChildNode.childNode(0).attr("title");
				String cateGoryCode = nextChildNode.childNode(0).attr("href").substring(11);

				cateGoryMap.put(cateGoryName, cateGoryCode);
			}
		}

		return cateGoryMap;
	}

	/**
	 * 获取百度阅读书籍分类Code及父类型 解决多种类型中含有"其他",以完成区分
	 * 
	 * @param url
	 * @throws IOException
	 */
	// @Deprecated
	// private void getBookCateGory(String url) throws IOException {
	// String appCategroyHtml = HtmlParseUtil.getHtmlDoc(url, ChartSet.GB2312);
	// Document doc = Jsoup.parse(appCategroyHtml);
	// Elements els = doc.select("div.popup-cate a[href^=/book/list]");
	// Iterator<Element> iterEls = els.iterator();
	// while (iterEls.hasNext()) {
	// Element nextEle = iterEls.next();
	// // System.out.println(nextEle);
	// System.out.print(nextEle.attr("title") + ":");
	// System.out.println(nextEle.attr("href").substring(11));
	// }
	//
	// }

}
